{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERTServiceForFFNNConversions.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "ZQRTFx5VamgN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Instructions\n",
        "\n",
        "# Open this and the other notebook (TextToBertEmbedToFFNNEmbed.ipynb)\n",
        "\n",
        "# Do not run anything yet\n",
        "\n",
        "# make sure both notebooks are on GPU mode. Make sure they are both connected, the top right of the notebook should NOT read 'busy' or 'connecting'. \n",
        "#Note: These notebooks won't work in CPU or TPU mode (perhaps there's a workaround)\n",
        "\n",
        "#Run the first line of code in the other notebook. DO NOT INSTALL TF 2.0 IN THE OTHER NOTEBOOK YET otherwise Bert as a Service won't run\n",
        "\n",
        "# Run all the code in this notebook \n",
        "\n",
        "# Look at the output for the last cell in this notebook. VERY IMPORTANT: Will until the output of this cell says something like:\n",
        "# \"\n",
        "# I:WORKER-0:[__i:gen:542]:ready and listening!\n",
        "# I:WORKER-2:[__i:gen:542]:ready and listening!\n",
        "# I:WORKER-3:[__i:gen:542]:ready and listening!\n",
        "# I:WORKER-1:[__i:gen:542]:ready and listening!\n",
        "#I:VENTILATOR:[__i:_ru:163]:all set, ready to serve request!\n",
        "#             \"\n",
        "#Wait, 10 more seconds, and then go to to the 2nd notebook\n",
        "\n",
        "# Finally, run all the cells in the other notebook, including installing TF 2.0 "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GJYsodX6arRH",
        "colab_type": "code",
        "outputId": "3c888404-4da3-4925-a43b-fcd9573f228b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install bert-serving-server  \n",
        "!pip install bert-serving-client  "
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting bert-serving-server\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d0/37/02bcb4b1508f4bd44b80d2e718d22e43f49c5b7b8f675a3918b4bd8519f8/bert_serving_server-1.8.9-py3-none-any.whl (60kB)\n",
            "\r\u001b[K    17% |█████▌                          | 10kB 22.0MB/s eta 0:00:01\r\u001b[K    34% |███████████                     | 20kB 2.1MB/s eta 0:00:01\r\u001b[K    51% |████████████████▍               | 30kB 3.1MB/s eta 0:00:01\r\u001b[K    68% |█████████████████████▉          | 40kB 2.0MB/s eta 0:00:01\r\u001b[K    85% |███████████████████████████▎    | 51kB 2.5MB/s eta 0:00:01\r\u001b[K    100% |████████████████████████████████| 61kB 2.8MB/s \n",
            "\u001b[?25hCollecting GPUtil>=1.3.0 (from bert-serving-server)\n",
            "  Downloading https://files.pythonhosted.org/packages/ed/0e/5c61eedde9f6c87713e89d794f01e378cfd9565847d4576fa627d758c554/GPUtil-1.4.0.tar.gz\n",
            "Collecting pyzmq>=17.1.0 (from bert-serving-server)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5f/04/f6f0fa20b698b29c6e6b1d6b4b575c12607b0abf61810aab1df4099988c6/pyzmq-18.0.1-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K    100% |████████████████████████████████| 1.1MB 15.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from bert-serving-server) (1.16.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from bert-serving-server) (1.12.0)\n",
            "Requirement already satisfied: termcolor>=1.1 in /usr/local/lib/python3.6/dist-packages (from bert-serving-server) (1.1.0)\n",
            "Building wheels for collected packages: GPUtil\n",
            "  Building wheel for GPUtil (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/3d/77/07/80562de4bb0786e5ea186911a2c831fdd0018bda69beab71fd\n",
            "Successfully built GPUtil\n",
            "Installing collected packages: GPUtil, pyzmq, bert-serving-server\n",
            "  Found existing installation: pyzmq 17.0.0\n",
            "    Uninstalling pyzmq-17.0.0:\n",
            "      Successfully uninstalled pyzmq-17.0.0\n",
            "Successfully installed GPUtil-1.4.0 bert-serving-server-1.8.9 pyzmq-18.0.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "zmq"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Collecting bert-serving-client\n",
            "  Downloading https://files.pythonhosted.org/packages/96/ed/23f3d7c1f897e09944c16d066e0e52bb83d0d94fdfe0dc9bb53ce1810ca1/bert_serving_client-1.8.9-py2.py3-none-any.whl\n",
            "Requirement already satisfied: pyzmq>=17.1.0 in /usr/local/lib/python3.6/dist-packages (from bert-serving-client) (18.0.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from bert-serving-client) (1.16.3)\n",
            "Installing collected packages: bert-serving-client\n",
            "Successfully installed bert-serving-client-1.8.9\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "q68GSedlawkP",
        "colab_type": "code",
        "outputId": "833d6012-f217-45a6-9989-2d54e8ccd74c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "import urllib.request\n",
        "...\n",
        "# Download the file from `url` and save it locally under `file_name`:\n",
        "urllib.request.urlretrieve('https://github.com/naver/biobert-pretrained/releases/download/v1.0-pubmed-pmc/biobert_pubmed_pmc.tar.gz', 'BioBert.tar.gz')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('BioBert.tar.gz', <http.client.HTTPMessage at 0x7f1e6b403cf8>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "metadata": {
        "id": "ZiJKbi7Te9k5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import os"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_-eSqByeazpv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "if not os.path.exists('BioBertFolder'):\n",
        "    os.makedirs('BioBertFolder')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XMcGddKUa09_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import tarfile\n",
        "tar = tarfile.open(\"BioBert.tar.gz\")\n",
        "tar.extractall(path='BioBertFolder/')\n",
        "tar.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pQPpKkFPa5Df",
        "colab_type": "code",
        "outputId": "8a6c7c50-7b57-49a5-dd1d-a4ccb08d05b4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        }
      },
      "cell_type": "code",
      "source": [
        "os.listdir('BioBertFolder/pubmed_pmc_470k')"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['vocab.txt',\n",
              " 'biobert_model.ckpt.index',\n",
              " 'biobert_model.ckpt.meta',\n",
              " '._bert_config.json',\n",
              " 'bert_config.json',\n",
              " 'biobert_model.ckpt.data-00000-of-00001']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "metadata": {
        "id": "cmORN1T8a6wf",
        "colab_type": "code",
        "outputId": "430e4867-2474-4b9d-ae16-adc5834a967d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1410
        }
      },
      "cell_type": "code",
      "source": [
        "#Make sure other notebook is connected to GPU before runing this\n",
        "\n",
        "!bert-serving-start -model_dir BioBertFolder/pubmed_pmc_470k/ -tuned_model_dir=BioBertFolder/pubmed_pmc_470k/ -ckpt_name=biobert_model.ckpt -num_worker=4 -max_seq_len=512\n",
        "\n",
        "# Look at the output for the last cell in this notebook. VERY IMPORTANT: Will until the output of this cell says something like:\n",
        "# \"\n",
        "# I:WORKER-0:[__i:gen:542]:ready and listening!\n",
        "# I:WORKER-2:[__i:gen:542]:ready and listening!\n",
        "# I:WORKER-3:[__i:gen:542]:ready and listening!\n",
        "# I:WORKER-1:[__i:gen:542]:ready and listening!\n",
        "#             \"\n",
        "#Wait, 10 more seconds, and then go to to the 2nd notebook\n",
        "\n",
        "# Finally, you can install TF 2.0 and run all the cells in the other notebook. "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "usage: /usr/local/bin/bert-serving-start -model_dir BioBertFolder/pubmed_pmc_470k/ -tuned_model_dir=BioBertFolder/pubmed_pmc_470k/ -ckpt_name=biobert_model.ckpt -num_worker=4 -max_seq_len=512\n",
            "                 ARG   VALUE\n",
            "__________________________________________________\n",
            "           ckpt_name = biobert_model.ckpt\n",
            "         config_name = bert_config.json\n",
            "                cors = *\n",
            "                 cpu = False\n",
            "          device_map = []\n",
            "  fixed_embed_length = False\n",
            "                fp16 = False\n",
            " gpu_memory_fraction = 0.5\n",
            "       graph_tmp_dir = None\n",
            "    http_max_connect = 10\n",
            "           http_port = None\n",
            "        mask_cls_sep = False\n",
            "      max_batch_size = 256\n",
            "         max_seq_len = 512\n",
            "           model_dir = BioBertFolder/pubmed_pmc_470k/\n",
            "          num_worker = 4\n",
            "       pooling_layer = [-2]\n",
            "    pooling_strategy = REDUCE_MEAN\n",
            "                port = 5555\n",
            "            port_out = 5556\n",
            "       prefetch_size = 10\n",
            " priority_batch_size = 16\n",
            "show_tokens_to_client = False\n",
            "     tuned_model_dir = BioBertFolder/pubmed_pmc_470k/\n",
            "             verbose = False\n",
            "                 xla = False\n",
            "\n",
            "I:\u001b[35mVENTILATOR\u001b[0m:[__i:__i: 66]:freeze, optimize and export graph, could take a while...\n",
            "I:\u001b[36mGRAPHOPT\u001b[0m:[gra:opt: 52]:model config: BioBertFolder/pubmed_pmc_470k/bert_config.json\n",
            "I:\u001b[36mGRAPHOPT\u001b[0m:[gra:opt: 55]:checkpoint (override by the fine-tuned model): BioBertFolder/pubmed_pmc_470k/biobert_model.ckpt\n",
            "I:\u001b[36mGRAPHOPT\u001b[0m:[gra:opt: 59]:build graph...\n",
            "\n",
            "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "I:\u001b[36mGRAPHOPT\u001b[0m:[gra:opt:128]:load parameters from checkpoint...\n",
            "I:\u001b[36mGRAPHOPT\u001b[0m:[gra:opt:132]:optimize...\n",
            "I:\u001b[36mGRAPHOPT\u001b[0m:[gra:opt:140]:freeze...\n",
            "I:\u001b[36mGRAPHOPT\u001b[0m:[gra:opt:145]:write graph to a tmp file: /tmp/tmpp3724t42\n",
            "I:\u001b[35mVENTILATOR\u001b[0m:[__i:__i: 74]:optimized graph is stored at: /tmp/tmpp3724t42\n",
            "I:\u001b[35mVENTILATOR\u001b[0m:[__i:_ru:128]:bind all sockets\n",
            "I:\u001b[35mVENTILATOR\u001b[0m:[__i:_ru:132]:open 8 ventilator-worker sockets\n",
            "I:\u001b[35mVENTILATOR\u001b[0m:[__i:_ru:135]:start the sink\n",
            "I:\u001b[35mVENTILATOR\u001b[0m:[__i:_ge:219]:get devices\n",
            "I:\u001b[32mSINK\u001b[0m:[__i:_ru:303]:ready\n",
            "W:\u001b[35mVENTILATOR\u001b[0m:[__i:_ge:234]:only 1 out of 1 GPU(s) is available/free, but \"-num_worker=4\"\n",
            "W:\u001b[35mVENTILATOR\u001b[0m:[__i:_ge:236]:multiple workers will be allocated to one GPU, may not scale well and may raise out-of-memory\n",
            "I:\u001b[35mVENTILATOR\u001b[0m:[__i:_ge:252]:device map: \n",
            "\t\tworker  0 -> gpu  0\n",
            "\t\tworker  1 -> gpu  0\n",
            "\t\tworker  2 -> gpu  0\n",
            "\t\tworker  3 -> gpu  0\n",
            "I:\u001b[33mWORKER-0\u001b[0m:[__i:_ru:514]:use device gpu: 0, load graph from /tmp/tmpp3724t42\n",
            "I:\u001b[33mWORKER-1\u001b[0m:[__i:_ru:514]:use device gpu: 0, load graph from /tmp/tmpp3724t42\n",
            "I:\u001b[33mWORKER-2\u001b[0m:[__i:_ru:514]:use device gpu: 0, load graph from /tmp/tmpp3724t42\n",
            "I:\u001b[33mWORKER-3\u001b[0m:[__i:_ru:514]:use device gpu: 0, load graph from /tmp/tmpp3724t42\n",
            "I:\u001b[33mWORKER-3\u001b[0m:[__i:gen:542]:ready and listening!\n",
            "I:\u001b[33mWORKER-2\u001b[0m:[__i:gen:542]:ready and listening!\n",
            "I:\u001b[33mWORKER-1\u001b[0m:[__i:gen:542]:ready and listening!\n",
            "I:\u001b[33mWORKER-0\u001b[0m:[__i:gen:542]:ready and listening!\n",
            "I:\u001b[35mVENTILATOR\u001b[0m:[__i:_ru:163]:all set, ready to serve request!\n",
            "I:\u001b[35mVENTILATOR\u001b[0m:[__i:_ru:177]:new config request\treq id: 1\tclient: b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d'\n",
            "I:\u001b[32mSINK\u001b[0m:[__i:_ru:354]:send config\tclient b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d'\n",
            "I:\u001b[35mVENTILATOR\u001b[0m:[__i:_ru:193]:new encode request\treq id: 2\tsize: 1\tclient: b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d'\n",
            "I:\u001b[32mSINK\u001b[0m:[__i:_ru:351]:job register\tsize: 1\tjob id: b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d#2'\n",
            "I:\u001b[33mWORKER-0\u001b[0m:[__i:gen:551]:new job\tsocket: 0\tsize: 1\tclient: b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d#2'\n",
            "I:\u001b[33mWORKER-0\u001b[0m:[__i:_ru:526]:job done\tsize: (1, 768)\tclient: b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d#2'\n",
            "I:\u001b[32mSINK\u001b[0m:[__i:_ru:331]:collect b'EMBEDDINGS' b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d#2' (E:1/T:0/A:1)\n",
            "I:\u001b[32mSINK\u001b[0m:[__i:_ru:340]:send back\tsize: 1\tjob id: b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d#2'\n",
            "I:\u001b[35mVENTILATOR\u001b[0m:[__i:_ru:193]:new encode request\treq id: 3\tsize: 1\tclient: b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d'\n",
            "I:\u001b[32mSINK\u001b[0m:[__i:_ru:351]:job register\tsize: 1\tjob id: b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d#3'\n",
            "I:\u001b[33mWORKER-2\u001b[0m:[__i:gen:551]:new job\tsocket: 0\tsize: 1\tclient: b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d#3'\n",
            "I:\u001b[33mWORKER-2\u001b[0m:[__i:_ru:526]:job done\tsize: (1, 768)\tclient: b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d#3'\n",
            "I:\u001b[32mSINK\u001b[0m:[__i:_ru:331]:collect b'EMBEDDINGS' b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d#3' (E:1/T:0/A:1)\n",
            "I:\u001b[32mSINK\u001b[0m:[__i:_ru:340]:send back\tsize: 1\tjob id: b'b34f8584-a1a6-44e0-bd27-0bf7ff2fcb4d#3'\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}